package com.bfmet.crawler;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.Div;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;

import com.bfmet.entity.News;
import com.bfmet.entity.News.Type;

public class PolicySpider extends Spider {

	@Override
	public void run() {
		this.getPolicy("http://www.cnmn.com.cn/ShowNewsList.aspx?id=13");
	}

	private void getPolicy(String url) {
		log("Fetch policy begin..." + url);
		try {
			Parser p = new Parser(url);

			NodeList ulList = p.parse(new AndFilter(new TagNameFilter("ul"),
					new HasAttributeFilter("class", "listul")));
			p.setInputHTML(ulList.elementAt(0).toHtml());
			NodeList liList = p.parse(new TagNameFilter("li"));
			for (Node node : liList.toNodeArray()) {
				if (node.getFirstChild() instanceof LinkTag) {
					LinkTag a = (LinkTag) node.getFirstChild();
					this.log("name:" + a.toPlainTextString());
					this.log("link:" + a.getLink());
					this.getPolicyContent(url.substring(0,
							url.lastIndexOf("/") + 1) + a.getLink());
				}
			}

		} catch (Exception ex) {
			this.logException(ex);
		}
		log("Fetch policy end..." + url);
	}

	private void getPolicyContent(String subUrl) {
		try {
			log("Fetch policy subUrl begin..." + subUrl);
			News news = new News();
			NodeFilter divf = new TagNameFilter("div");
			NodeFilter newsContent = new HasAttributeFilter("id", "newsContent");

			Parser p = new Parser(subUrl);
			NodeList nodeList = p.parse(new AndFilter(divf, newsContent));
			if (nodeList.elementAt(0) instanceof Div) {
				Div div = (Div) nodeList.elementAt(0);
				String title = div.getFirstChild().getNextSibling()
						.toPlainTextString();
				this.log("policy title:" + title);
				news.setName(title);
				p.setInputHTML(div.toHtml());
				NodeList nl = p.parse(new AndFilter(divf,
						new HasAttributeFilter("id", "content")));
				if (nl.elementAt(0) instanceof Div) {
					Div divContent = (Div) nl.elementAt(0);
					String content = divContent.toPlainTextString();
					this.log("policy content:" + content);
					news.setContent(content);
				}
			}
			news.setType(Type.POLICY);
			this.save(news);
		} catch (Exception ex) {
			this.logException(ex);
		}
		log("Fetch policy subUrl end..." + subUrl);
	}

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		PolicySpider ps = new PolicySpider();
		ps.run();
	}

}
